'''
Created on 2014-3-6

@author: xizhzhao
'''
from lxml.html import *
import datetime
from core.PageFetcher import *
from xparser.TemplateParser import *
from dboperater.DbOperatorFactory import *

class TemplatePageFetcher(PageFetcher):
    def init(self, url, tmplt_name, intent=False):
        # first should get template from
        #  "carhrdb"
        self.__db_key__ = ""
        if isinstance(intent, dict):
            try:
                self.__db_key__ = intent['db_key']
            except:
                pass

        domain = url.split('/')[2]
        tmplt_file = Configurator.get_template(domain, tmplt_name)
        tmplt_file = ''.join(tmplt_file).strip()

        if tmplt_file == '':
            self.__template_name__ = tmplt_name
        else:
            self.__template_name__ = tmplt_file

        self.__template_path__ = Configurator.get_val('page_template_path')
        tmplt_file = ('%s/%s.xml') % (self.__template_path__, self.__template_name__)
        logger.error('checking template file: %s'% tmplt_file )

        import os
        if os.path.exists(tmplt_file) and not(os.path.getsize(tmplt_file) == 0):
            self.__template__ = TemplateParser(tmplt_file)
            if not self.__template__.do_parse():
                return False
            factory = DbOperatorFactory()
            self.__db__ = factory.GetDbOperator(self.__db_key__)
            return super(TemplatePageFetcher, self).init(url, False)
        else:
            self.__url__ = url
            err_txt = ('No template exist for %s') % (tmplt_name)
            self.__write_to_uncrawled_file(err_txt)
            logger.error(err_txt)
            return False
    
    def __write_to_uncrawled_file(self, err_txt):
        uncrawled_webs = Configurator.get_val('uncrawled_websites_lib')
        mutex = threading.Lock()
        if mutex.acquire(1):
            f = open(("%s/UnCrawledWebList.csv") % (uncrawled_webs), 'a')
            import time
            
            info = "%s,%s,%s,%s,%s,%s" % (self.__template_name__, self.__url__, Configurator.get_val('key_word'), err_txt, time.strftime('%Y-%m-%d', time.localtime(time.time())), '\n')
            f.write(info)
            f.close()
            mutex.release()

    def __fetch_html(self, url):
        if self.__proxy_needed__:
            proxy_handler = urllib2.ProxyHandler({Configurator.get_val('proxy_type') : Configurator.get_val('proxy_url')})
            opener = urllib2.build_opener(proxy_handler, urllib2.HTTPHandler)
            urllib2.install_opener(opener)
        
        reader = urllib2.urlopen(url)
        return reader.read()
    
    def __get_parsed_html_instance(self, ref_key):
        if self.__template__.is_com_foreign_key(ref_key):
            return self.__parsed_com_html__
        if self.__template__.is_job_foreign_key(ref_key):
            return self.__parsed_job_html__
        if self.__template__.is_time_foreign_key(ref_key):
            return self.__parsed_time_html__
        
        return self.__parsed_html__
    
    def __get_link_url__(self, x_path):
        if not x_path.endswith('@href'):
            x_path = x_path + '/@href'
        
        base_url = self.__parsed_html__.xpath(self.__template__.get_com_link())
        if isinstance(base_url, list):
            if len(base_url) > 0:
                base_url = base_url[0]
            else:
                return None
        
        if not PageFetcher.is_legal_url(base_url):
            url = PageFetcher.get_legal_url(self.__url__, base_url)
            if PageFetcher.is_legal_url(url):
                return url
        
        return None
    
    def __prepare_parsed_html_instances(self):
        from lxml.html import fromstring
        self.__parsed_html__ = fromstring(self.__html_src__)
        ''' Fetch the foreign page first'''
        self.__parsed_com_html__ = None
        self.__parsed_job_html__ = None
        self.__parsed_time_html__ = None
        if self.__template__.has_foreign_link():
            if not self.__template__.get_com_link() == 'Null':
                url = self.__get_link_url__(self.__template__.get_com_link())
                if not (url == None):
                    self.__parsed_com_html__ = fromstring(self.__fetch_html(url))
            if not self.__template__.get_job_link() == 'Null':
                url = self.__get_link_url__(self.__template__.get_job_link())
                if not (url == None):
                    self.__parsed_job_html__ = fromstring(self.__fetch_html(url))
            if not self.__template__.get_time_link() == 'Null':
                url = self.__get_link_url__(self.__template__.get_time_link())
                if not (url == None):
                    self.__parsed_time_html__ = fromstring(self.__fetch_html(url))

    def analysis(self):
        self.__time__ = datetime.datetime.now()
        
        if self.__html_src__:
            self.__prepare_parsed_html_instances()
            for (key, xpath) in self.__template__.get_xpathes():
                if not xpath == 'Null' and not xpath == '':
                    parsed_html = self.__get_parsed_html_instance(key)
                    
                    if parsed_html == None:
                        err_txt = "%s : %s can not get html parsed!" % (self.__url__, self.__template_name__)
                        self.__write_to_uncrawled_file(err_txt)
                        logger.error(err_txt)
                        continue
                    logger.info('parsing key : %s...' % (key))

                    # add || or logic for choosing an alternative xpath template parse
                    xpathlist = []
                    if xpath.find("||") != -1:
                        xpathlist = xpath.split("||")

                    if len(xpathlist) == 0:
                        xpathlist.append(xpath)

                    for splitted_xpath in xpathlist:
                        x_content = parsed_html.xpath(splitted_xpath)

                        if isinstance(x_content, list):
                            if len(x_content) > 0:
                                x_content = x_content[0]
                            else:
                                x_content = ''  # give x_content a empty
                    
                        value = ''
                        from lxml.html import HtmlElement
                        if isinstance(x_content, HtmlElement):
                            value = x_content.text_content()
                        else:
                            value = x_content

                        if isinstance(value, unicode):
                            # solve the encoding error
                            removelist = [u'\\xa0', u'\\r', u'\\n', u' ', u'\\t', u'\\a', u'\\b', u'\\f', u'\\v', u'\\0']  # remove the special chars
                            t = value.encode('unicode_escape')
                            for rl in removelist:
                                t = t.replace(rl, u'')

                            start = 0
                            count = 0
                            while start < len(t):
                                i = t.find('\\', start, len(t))
                                if t[i + 1] == ('x') or t[i + 1] == ('X'):
                                    count = count + 1
                                if start > i:
                                    break
                                start = i + 1
                            if not t.count('\\') == 0:
                                rate = (float)(count) / (float)(t.count('\\'))
                                if rate > 0.5:
                                    value = t.decode('unicode_escape').encode('raw_unicode_escape')
                            else:
                                value = t.decode('unicode_escape').encode('utf8')

                            # remove the ' and "
                            for c in ['\'', '"']:
                                value = value.replace(c, '')

                        if value == '' or value == None: continue
                        self.__template__.insert_info(key, ''.join(value.split()))
                        break
            
            # insert into db
            record = self.__template__.genrate_record(Configurator.get_val('key_word'), self.__encoding__, self.__url__, self.__template_name__)

            # import chardet
            # print record.keyword
            # encoding1 = chardet.detect(self.__url__)['encoding']
            #
            # print encoding1
            db_record = self.__db__.query_record_use_fromurl(self.__url__, record.keyword)
            if db_record == None or not Record.compare_record(db_record, record):  # if not the same, return true 
                self.__db__.insert_record(record)
            else:
                logger.info(self.__url__ + "keeps the same as the latest one, nothing to insert !!")